<!DOCTYPE html>
<!-- saved from url=(0014)about:internet -->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
<meta http-equiv="x-ua-compatible" content="IE=9" >

<title>(String/text processing)++: stringi 0.2-3 released</title>

<style type="text/css">
body, td {
   font-family: sans-serif;
   background-color: white;
   font-size: 12px;
   margin: 8px;
}

tt, code, pre {
   font-family: 'DejaVu Sans Mono', 'Droid Sans Mono', 'Lucida Console', Consolas, Monaco, monospace;
}

h1 { 
   font-size:2.2em; 
}

h2 { 
   font-size:1.8em; 
}

h3 { 
   font-size:1.4em; 
}

h4 { 
   font-size:1.0em; 
}

h5 { 
   font-size:0.9em; 
}

h6 { 
   font-size:0.8em; 
}

a:visited {
   color: rgb(50%, 0%, 50%);
}

pre {	
   margin-top: 0;
   max-width: 95%;
   border: 1px solid #ccc;
   white-space: pre-wrap;
}

pre code {
   display: block; padding: 0.5em;
}

code.r, code.cpp {
   background-color: #F8F8F8;
}

table, td, th {
  border: none;
}

blockquote {
   color:#666666;
   margin:0;
   padding-left: 1em;
   border-left: 0.5em #EEE solid;
}

hr {
   height: 0px;
   border-bottom: none;
   border-top-width: thin;
   border-top-style: dotted;
   border-top-color: #999999;
}

@media print {
   * { 
      background: transparent !important; 
      color: black !important; 
      filter:none !important; 
      -ms-filter: none !important; 
   }

   body { 
      font-size:12pt; 
      max-width:100%; 
   }
       
   a, a:visited { 
      text-decoration: underline; 
   }

   hr { 
      visibility: hidden;
      page-break-before: always;
   }

   pre, blockquote { 
      padding-right: 1em; 
      page-break-inside: avoid; 
   }

   tr, img { 
      page-break-inside: avoid; 
   }

   img { 
      max-width: 100% !important; 
   }

   @page :left { 
      margin: 15mm 20mm 15mm 10mm; 
   }
     
   @page :right { 
      margin: 15mm 10mm 15mm 20mm; 
   }

   p, h2, h3 { 
      orphans: 3; widows: 3; 
   }

   h2, h3 { 
      page-break-after: avoid; 
   }
}

</style>

<!-- Styles for R syntax highlighter -->
<style type="text/css">
   pre .operator,
   pre .paren {
     color: rgb(104, 118, 135)
   }

   pre .literal {
     color: rgb(88, 72, 246)
   }

   pre .number {
     color: rgb(0, 0, 205);
   }

   pre .comment {
     color: rgb(76, 136, 107);
   }

   pre .keyword {
     color: rgb(0, 0, 255);
   }

   pre .identifier {
     color: rgb(0, 0, 0);
   }

   pre .string {
     color: rgb(3, 106, 7);
   }
</style>

<!-- R syntax highlighter -->
<script type="text/javascript">
var hljs=new function(){function m(p){return p.replace(/&/gm,"&amp;").replace(/</gm,"&lt;")}function f(r,q,p){return RegExp(q,"m"+(r.cI?"i":"")+(p?"g":""))}function b(r){for(var p=0;p<r.childNodes.length;p++){var q=r.childNodes[p];if(q.nodeName=="CODE"){return q}if(!(q.nodeType==3&&q.nodeValue.match(/\s+/))){break}}}function h(t,s){var p="";for(var r=0;r<t.childNodes.length;r++){if(t.childNodes[r].nodeType==3){var q=t.childNodes[r].nodeValue;if(s){q=q.replace(/\n/g,"")}p+=q}else{if(t.childNodes[r].nodeName=="BR"){p+="\n"}else{p+=h(t.childNodes[r])}}}if(/MSIE [678]/.test(navigator.userAgent)){p=p.replace(/\r/g,"\n")}return p}function a(s){var r=s.className.split(/\s+/);r=r.concat(s.parentNode.className.split(/\s+/));for(var q=0;q<r.length;q++){var p=r[q].replace(/^language-/,"");if(e[p]){return p}}}function c(q){var p=[];(function(s,t){for(var r=0;r<s.childNodes.length;r++){if(s.childNodes[r].nodeType==3){t+=s.childNodes[r].nodeValue.length}else{if(s.childNodes[r].nodeName=="BR"){t+=1}else{if(s.childNodes[r].nodeType==1){p.push({event:"start",offset:t,node:s.childNodes[r]});t=arguments.callee(s.childNodes[r],t);p.push({event:"stop",offset:t,node:s.childNodes[r]})}}}}return t})(q,0);return p}function k(y,w,x){var q=0;var z="";var s=[];function u(){if(y.length&&w.length){if(y[0].offset!=w[0].offset){return(y[0].offset<w[0].offset)?y:w}else{return w[0].event=="start"?y:w}}else{return y.length?y:w}}function t(D){var A="<"+D.nodeName.toLowerCase();for(var B=0;B<D.attributes.length;B++){var C=D.attributes[B];A+=" "+C.nodeName.toLowerCase();if(C.value!==undefined&&C.value!==false&&C.value!==null){A+='="'+m(C.value)+'"'}}return A+">"}while(y.length||w.length){var v=u().splice(0,1)[0];z+=m(x.substr(q,v.offset-q));q=v.offset;if(v.event=="start"){z+=t(v.node);s.push(v.node)}else{if(v.event=="stop"){var p,r=s.length;do{r--;p=s[r];z+=("</"+p.nodeName.toLowerCase()+">")}while(p!=v.node);s.splice(r,1);while(r<s.length){z+=t(s[r]);r++}}}}return z+m(x.substr(q))}function j(){function q(x,y,v){if(x.compiled){return}var u;var s=[];if(x.k){x.lR=f(y,x.l||hljs.IR,true);for(var w in x.k){if(!x.k.hasOwnProperty(w)){continue}if(x.k[w] instanceof Object){u=x.k[w]}else{u=x.k;w="keyword"}for(var r in u){if(!u.hasOwnProperty(r)){continue}x.k[r]=[w,u[r]];s.push(r)}}}if(!v){if(x.bWK){x.b="\\b("+s.join("|")+")\\s"}x.bR=f(y,x.b?x.b:"\\B|\\b");if(!x.e&&!x.eW){x.e="\\B|\\b"}if(x.e){x.eR=f(y,x.e)}}if(x.i){x.iR=f(y,x.i)}if(x.r===undefined){x.r=1}if(!x.c){x.c=[]}x.compiled=true;for(var t=0;t<x.c.length;t++){if(x.c[t]=="self"){x.c[t]=x}q(x.c[t],y,false)}if(x.starts){q(x.starts,y,false)}}for(var p in e){if(!e.hasOwnProperty(p)){continue}q(e[p].dM,e[p],true)}}function d(B,C){if(!j.called){j();j.called=true}function q(r,M){for(var L=0;L<M.c.length;L++){if((M.c[L].bR.exec(r)||[null])[0]==r){return M.c[L]}}}function v(L,r){if(D[L].e&&D[L].eR.test(r)){return 1}if(D[L].eW){var M=v(L-1,r);return M?M+1:0}return 0}function w(r,L){return L.i&&L.iR.test(r)}function K(N,O){var M=[];for(var L=0;L<N.c.length;L++){M.push(N.c[L].b)}var r=D.length-1;do{if(D[r].e){M.push(D[r].e)}r--}while(D[r+1].eW);if(N.i){M.push(N.i)}return f(O,M.join("|"),true)}function p(M,L){var N=D[D.length-1];if(!N.t){N.t=K(N,E)}N.t.lastIndex=L;var r=N.t.exec(M);return r?[M.substr(L,r.index-L),r[0],false]:[M.substr(L),"",true]}function z(N,r){var L=E.cI?r[0].toLowerCase():r[0];var M=N.k[L];if(M&&M instanceof Array){return M}return false}function F(L,P){L=m(L);if(!P.k){return L}var r="";var O=0;P.lR.lastIndex=0;var M=P.lR.exec(L);while(M){r+=L.substr(O,M.index-O);var N=z(P,M);if(N){x+=N[1];r+='<span class="'+N[0]+'">'+M[0]+"</span>"}else{r+=M[0]}O=P.lR.lastIndex;M=P.lR.exec(L)}return r+L.substr(O,L.length-O)}function J(L,M){if(M.sL&&e[M.sL]){var r=d(M.sL,L);x+=r.keyword_count;return r.value}else{return F(L,M)}}function I(M,r){var L=M.cN?'<span class="'+M.cN+'">':"";if(M.rB){y+=L;M.buffer=""}else{if(M.eB){y+=m(r)+L;M.buffer=""}else{y+=L;M.buffer=r}}D.push(M);A+=M.r}function G(N,M,Q){var R=D[D.length-1];if(Q){y+=J(R.buffer+N,R);return false}var P=q(M,R);if(P){y+=J(R.buffer+N,R);I(P,M);return P.rB}var L=v(D.length-1,M);if(L){var O=R.cN?"</span>":"";if(R.rE){y+=J(R.buffer+N,R)+O}else{if(R.eE){y+=J(R.buffer+N,R)+O+m(M)}else{y+=J(R.buffer+N+M,R)+O}}while(L>1){O=D[D.length-2].cN?"</span>":"";y+=O;L--;D.length--}var r=D[D.length-1];D.length--;D[D.length-1].buffer="";if(r.starts){I(r.starts,"")}return R.rE}if(w(M,R)){throw"Illegal"}}var E=e[B];var D=[E.dM];var A=0;var x=0;var y="";try{var s,u=0;E.dM.buffer="";do{s=p(C,u);var t=G(s[0],s[1],s[2]);u+=s[0].length;if(!t){u+=s[1].length}}while(!s[2]);if(D.length>1){throw"Illegal"}return{r:A,keyword_count:x,value:y}}catch(H){if(H=="Illegal"){return{r:0,keyword_count:0,value:m(C)}}else{throw H}}}function g(t){var p={keyword_count:0,r:0,value:m(t)};var r=p;for(var q in e){if(!e.hasOwnProperty(q)){continue}var s=d(q,t);s.language=q;if(s.keyword_count+s.r>r.keyword_count+r.r){r=s}if(s.keyword_count+s.r>p.keyword_count+p.r){r=p;p=s}}if(r.language){p.second_best=r}return p}function i(r,q,p){if(q){r=r.replace(/^((<[^>]+>|\t)+)/gm,function(t,w,v,u){return w.replace(/\t/g,q)})}if(p){r=r.replace(/\n/g,"<br>")}return r}function n(t,w,r){var x=h(t,r);var v=a(t);var y,s;if(v){y=d(v,x)}else{return}var q=c(t);if(q.length){s=document.createElement("pre");s.innerHTML=y.value;y.value=k(q,c(s),x)}y.value=i(y.value,w,r);var u=t.className;if(!u.match("(\\s|^)(language-)?"+v+"(\\s|$)")){u=u?(u+" "+v):v}if(/MSIE [678]/.test(navigator.userAgent)&&t.tagName=="CODE"&&t.parentNode.tagName=="PRE"){s=t.parentNode;var p=document.createElement("div");p.innerHTML="<pre><code>"+y.value+"</code></pre>";t=p.firstChild.firstChild;p.firstChild.cN=s.cN;s.parentNode.replaceChild(p.firstChild,s)}else{t.innerHTML=y.value}t.className=u;t.result={language:v,kw:y.keyword_count,re:y.r};if(y.second_best){t.second_best={language:y.second_best.language,kw:y.second_best.keyword_count,re:y.second_best.r}}}function o(){if(o.called){return}o.called=true;var r=document.getElementsByTagName("pre");for(var p=0;p<r.length;p++){var q=b(r[p]);if(q){n(q,hljs.tabReplace)}}}function l(){if(window.addEventListener){window.addEventListener("DOMContentLoaded",o,false);window.addEventListener("load",o,false)}else{if(window.attachEvent){window.attachEvent("onload",o)}else{window.onload=o}}}var e={};this.LANGUAGES=e;this.highlight=d;this.highlightAuto=g;this.fixMarkup=i;this.highlightBlock=n;this.initHighlighting=o;this.initHighlightingOnLoad=l;this.IR="[a-zA-Z][a-zA-Z0-9_]*";this.UIR="[a-zA-Z_][a-zA-Z0-9_]*";this.NR="\\b\\d+(\\.\\d+)?";this.CNR="\\b(0[xX][a-fA-F0-9]+|(\\d+(\\.\\d*)?|\\.\\d+)([eE][-+]?\\d+)?)";this.BNR="\\b(0b[01]+)";this.RSR="!|!=|!==|%|%=|&|&&|&=|\\*|\\*=|\\+|\\+=|,|\\.|-|-=|/|/=|:|;|<|<<|<<=|<=|=|==|===|>|>=|>>|>>=|>>>|>>>=|\\?|\\[|\\{|\\(|\\^|\\^=|\\||\\|=|\\|\\||~";this.ER="(?![\\s\\S])";this.BE={b:"\\\\.",r:0};this.ASM={cN:"string",b:"'",e:"'",i:"\\n",c:[this.BE],r:0};this.QSM={cN:"string",b:'"',e:'"',i:"\\n",c:[this.BE],r:0};this.CLCM={cN:"comment",b:"//",e:"$"};this.CBLCLM={cN:"comment",b:"/\\*",e:"\\*/"};this.HCM={cN:"comment",b:"#",e:"$"};this.NM={cN:"number",b:this.NR,r:0};this.CNM={cN:"number",b:this.CNR,r:0};this.BNM={cN:"number",b:this.BNR,r:0};this.inherit=function(r,s){var p={};for(var q in r){p[q]=r[q]}if(s){for(var q in s){p[q]=s[q]}}return p}}();hljs.LANGUAGES.cpp=function(){var a={keyword:{"false":1,"int":1,"float":1,"while":1,"private":1,"char":1,"catch":1,"export":1,virtual:1,operator:2,sizeof:2,dynamic_cast:2,typedef:2,const_cast:2,"const":1,struct:1,"for":1,static_cast:2,union:1,namespace:1,unsigned:1,"long":1,"throw":1,"volatile":2,"static":1,"protected":1,bool:1,template:1,mutable:1,"if":1,"public":1,friend:2,"do":1,"return":1,"goto":1,auto:1,"void":2,"enum":1,"else":1,"break":1,"new":1,extern:1,using:1,"true":1,"class":1,asm:1,"case":1,typeid:1,"short":1,reinterpret_cast:2,"default":1,"double":1,register:1,explicit:1,signed:1,typename:1,"try":1,"this":1,"switch":1,"continue":1,wchar_t:1,inline:1,"delete":1,alignof:1,char16_t:1,char32_t:1,constexpr:1,decltype:1,noexcept:1,nullptr:1,static_assert:1,thread_local:1,restrict:1,_Bool:1,complex:1},built_in:{std:1,string:1,cin:1,cout:1,cerr:1,clog:1,stringstream:1,istringstream:1,ostringstream:1,auto_ptr:1,deque:1,list:1,queue:1,stack:1,vector:1,map:1,set:1,bitset:1,multiset:1,multimap:1,unordered_set:1,unordered_map:1,unordered_multiset:1,unordered_multimap:1,array:1,shared_ptr:1}};return{dM:{k:a,i:"</",c:[hljs.CLCM,hljs.CBLCLM,hljs.QSM,{cN:"string",b:"'\\\\?.",e:"'",i:"."},{cN:"number",b:"\\b(\\d+(\\.\\d*)?|\\.\\d+)(u|U|l|L|ul|UL|f|F)"},hljs.CNM,{cN:"preprocessor",b:"#",e:"$"},{cN:"stl_container",b:"\\b(deque|list|queue|stack|vector|map|set|bitset|multiset|multimap|unordered_map|unordered_set|unordered_multiset|unordered_multimap|array)\\s*<",e:">",k:a,r:10,c:["self"]}]}}}();hljs.LANGUAGES.r={dM:{c:[hljs.HCM,{cN:"number",b:"\\b0[xX][0-9a-fA-F]+[Li]?\\b",e:hljs.IMMEDIATE_RE,r:0},{cN:"number",b:"\\b\\d+(?:[eE][+\\-]?\\d*)?L\\b",e:hljs.IMMEDIATE_RE,r:0},{cN:"number",b:"\\b\\d+\\.(?!\\d)(?:i\\b)?",e:hljs.IMMEDIATE_RE,r:1},{cN:"number",b:"\\b\\d+(?:\\.\\d*)?(?:[eE][+\\-]?\\d*)?i?\\b",e:hljs.IMMEDIATE_RE,r:0},{cN:"number",b:"\\.\\d+(?:[eE][+\\-]?\\d*)?i?\\b",e:hljs.IMMEDIATE_RE,r:1},{cN:"keyword",b:"(?:tryCatch|library|setGeneric|setGroupGeneric)\\b",e:hljs.IMMEDIATE_RE,r:10},{cN:"keyword",b:"\\.\\.\\.",e:hljs.IMMEDIATE_RE,r:10},{cN:"keyword",b:"\\.\\.\\d+(?![\\w.])",e:hljs.IMMEDIATE_RE,r:10},{cN:"keyword",b:"\\b(?:function)",e:hljs.IMMEDIATE_RE,r:2},{cN:"keyword",b:"(?:if|in|break|next|repeat|else|for|return|switch|while|try|stop|warning|require|attach|detach|source|setMethod|setClass)\\b",e:hljs.IMMEDIATE_RE,r:1},{cN:"literal",b:"(?:NA|NA_integer_|NA_real_|NA_character_|NA_complex_)\\b",e:hljs.IMMEDIATE_RE,r:10},{cN:"literal",b:"(?:NULL|TRUE|FALSE|T|F|Inf|NaN)\\b",e:hljs.IMMEDIATE_RE,r:1},{cN:"identifier",b:"[a-zA-Z.][a-zA-Z0-9._]*\\b",e:hljs.IMMEDIATE_RE,r:0},{cN:"operator",b:"<\\-(?!\\s*\\d)",e:hljs.IMMEDIATE_RE,r:2},{cN:"operator",b:"\\->|<\\-",e:hljs.IMMEDIATE_RE,r:1},{cN:"operator",b:"%%|~",e:hljs.IMMEDIATE_RE},{cN:"operator",b:">=|<=|==|!=|\\|\\||&&|=|\\+|\\-|\\*|/|\\^|>|<|!|&|\\||\\$|:",e:hljs.IMMEDIATE_RE,r:0},{cN:"operator",b:"%",e:"%",i:"\\n",r:1},{cN:"identifier",b:"`",e:"`",r:0},{cN:"string",b:'"',e:'"',c:[hljs.BE],r:0},{cN:"string",b:"'",e:"'",c:[hljs.BE],r:0},{cN:"paren",b:"[[({\\])}]",e:hljs.IMMEDIATE_RE,r:0}]}};
hljs.initHighlightingOnLoad();
</script>




</head>

<body>
<h1>(String/text processing)++: stringi 0.2-3 released</h1>

<p>A new release of the <a href="http://stringi.rexamine.com"><code>stringi</code></a> package is
available on <a href="http://cran.r-project.org/web/packages/stringi/index.html">CRAN</a>
(please wait a few days for Windows and OS X binary builds).</p>

<p><strong><code>stringi</code> is a package providing</strong> (but definitely not limiting to) equivalents
of nearly all the <strong>character string processing functions</strong> known
from base R. While developing the package we had
<strong>high performance</strong> and <strong>portability</strong> of its facilities in our minds.</p>

<p><code>stringi</code>&#39;s user interface is <strong>inspired by
and consistent with</strong> that of Hadley Wickham&#39;s great
<strong><a href="http://cran.r-project.org/web/packages/stringr/index.html"><code>stringr</code></a> package</strong>.
Quoting its <a href="http://cran.r-project.org/web/packages/stringr/README.html">README</a>,
<code>stringr</code> (and &ldquo;hence&rdquo; <code>stringi</code>):</p>

<blockquote>
<ul>
<li>Processes factors and characters in the same way,</li>
<li>Gives functions consistent names and arguments,</li>
<li>Simplifies string operations by eliminating options that you don&#39;t need 95% of the time,</li>
<li>Produces outputs than can easily be used as inputs. This includes ensuring that missing inputs result in missing outputs, and zero length inputs result in zero length outputs,</li>
<li>Completes R&#39;s string handling functions with useful functions from other programming languages.</li>
</ul>
</blockquote>

<h3>Some problems with base R functions</h3>

<p>While base R as well as <code>stringr</code> functions are great for simple
text processing tasks, dealing with more complex ones
(such as natural language processing) may be a bit problematic.</p>

<p>First of all, some time ago we mentioned in
<a href="http://www.rexamine.com/2013/04/properly-internationalized-regular-expressions-in-r/">our blog post</a>
that regex search may provide <strong>different outputs on different platforms</strong>.
For example, Polish letters such as ą, ę, ś etc. are correctly
captured with <code>[[:alpha:]]</code> by the (default) ERE engine on Linux (native encoding=UTF-8),
while on Windows the results are quite surprising.
(A year ago my students got (of course, initially)
very bad marks from a Polish text processing
task just because they had written their R scripts on Windows while I
ran them on Linux.) :-)</p>

<p>Secondly, natural language processing relies on <strong>a set of very complex,
locale-specific rules</strong>. However, the rules available
(via e.g. <a href="http://www.gnu.org/software/libc/manual/html_node/Character-Handling.html">glibc</a>)
in base R string functions may sometimes give incorrect results.
For example, when we convert German <code>ß</code> (es-zett/double small s)
character to upper case, we rather expect <code>SS</code> in result than:</p>

<pre><code class="r">toupper(&quot;groß&quot;) # GROSS? No...
</code></pre>

<pre><code>## [1] &quot;GROß&quot;
</code></pre>

<p>Moreover, let&#39;s assume that we are asked to sort a character
vector according to the rules specific to the Slovak language.
Here, quite interestingly, the word <code>hladný</code> (hungry)
can be found in a dictionary <em>before</em> the word <code>chladný</code> (cold).
Of course, as not everyone works in a Slovak locale,
we don&#39;t expect to obtain a proper order immediately:</p>

<pre><code class="r">sort(c(&quot;hladný&quot;, &quot;chladný&quot;))
</code></pre>

<pre><code>## [1] &quot;chladný&quot; &quot;hladný&quot;
</code></pre>

<p>In order to obtain a proper order, we should temporarily
switch to a Slovak &ldquo;environment&rdquo;:</p>

<pre><code class="r">oldlocale &lt;- Sys.getlocale(&quot;LC_COLLATE&quot;)
Sys.setlocale(&quot;LC_COLLATE&quot;, &quot;sk_SK&quot;)
</code></pre>

<pre><code>## [1] &quot;sk_SK&quot;
</code></pre>

<pre><code class="r">sort(c(&quot;hladný&quot;, &quot;chladný&quot;))
</code></pre>

<pre><code>## [1] &quot;hladný&quot;  &quot;chladný&quot;
</code></pre>

<pre><code class="r">Sys.setlocale(&quot;LC_COLLATE&quot;, oldlocale)
</code></pre>

<pre><code>## [1] &quot;pl_PL.UTF-8&quot;
</code></pre>

<p>This code works on my Linux, but is <strong>not portable</strong>. It&#39;s because:</p>

<ol>
<li>Other Linux users may not have Slovak rule-base installed (and
not everyone has abilities to do it on his/her own).</li>
<li>Windows users don&#39;t use BCP 47-based locale names. There,
LCID <code>Slovak_Slovakia.1250</code> is appropriate.</li>
</ol>

<p>And so on.</p>

<h3><code>stringi</code> facilities</h3>

<p>In order to overcome such problems we decided to reimplement
each string processing function from scratch
(of course, purely in C++). The internationalization and globalization
support, as well as many string processing facilities
(like regex searching) is guaranteed by the well-known and established
<strong>IBM&#39;s <a href="http://site.icu-project.org/">ICU4C</a> library</strong> (refer to
ICU&#39;s website for more details).</p>

<p>Here is a very general list of the <strong>most important
features</strong> available in the current version of <code>stringi</code>:</p>

<ul>
<li>string searching:

<ul>
<li>with ICU (Java-like) regular expressions,</li>
<li>ICU USearch-based locale-aware string searching
(quite slow, but working properly e.g. for non-Unicode normalized strings),</li>
<li>very fast, locale-independent byte-wise pattern matching;</li>
</ul></li>
<li>joining and duplicating strings;</li>
<li>extracting and replacing substrings;</li>
<li>string trimming, padding, and text wrapping (e.g. with Knuth&#39;s dynamic word wrap algorithm);</li>
<li>text transliteration;</li>
<li>text collation (comparing, sorting);</li>
<li>text boundary analysis (e.g. for extracting individual words);</li>
<li>random string generation;</li>
<li>Unicode normalization;</li>
<li>character encoding conversion and detection;</li>
</ul>

<p>and many more.</p>

<h3>Showcase</h3>

<p>Here&#39;s a bunch of examples.</p>

<pre><code class="r">library(stringi)
</code></pre>

<ul>
<li>Proper <code>NA</code> handling:</li>
</ul>

<pre><code class="r">stri_length(c(&quot;aaa&quot;, NA, &quot;&quot;))
</code></pre>

<pre><code>## [1]  3 NA  0
</code></pre>

<ul>
<li>&ldquo;Deep&rdquo; vectorization:</li>
</ul>

<pre><code class="r">stri_replace_all_fixed(c(&quot;aba&quot;, &quot;bab&quot;), c(&quot;a&quot;, &quot;b&quot;), c(&quot;c&quot;, &quot;d&quot;))  # 1-1-1 and 2-2-2
</code></pre>

<pre><code>## [1] &quot;cbc&quot; &quot;dad&quot;
</code></pre>

<pre><code class="r">stri_replace_all_fixed(c(&quot;aba&quot;, &quot;bab&quot;), &quot;a&quot;, &quot;c&quot;)  # 1-1-1 and 2-1-1
</code></pre>

<pre><code>## [1] &quot;cbc&quot; &quot;bcb&quot;
</code></pre>

<pre><code class="r">stri_replace_all_fixed(&quot;aba&quot;, c(&quot;a&quot;, &quot;b&quot;), &quot;c&quot;)  # 1-1-1 and 1-2-1
</code></pre>

<pre><code>## [1] &quot;cbc&quot; &quot;aca&quot;
</code></pre>

<pre><code class="r">stri_replace_all_fixed(&quot;aba&quot;, &quot;a&quot;, c(&quot;c&quot;, &quot;d&quot;))  # 1-1-1 and 1-1-2
</code></pre>

<pre><code>## [1] &quot;cbc&quot; &quot;dbd&quot;
</code></pre>

<p>(all the functions are vectorized w.r.t most of their arguments)</p>

<ul>
<li>Easy-to-use, portable locale selection:</li>
</ul>

<pre><code class="r">stri_sort(c(&quot;hladný&quot;, &quot;chladný&quot;), opts = stri_opts_collator(locale = &quot;sk_SK&quot;))
</code></pre>

<pre><code>## [1] &quot;hladný&quot;  &quot;chladný&quot;
</code></pre>

<ul>
<li>Proper transliteration rules:</li>
</ul>

<pre><code class="r">stri_trans_toupper(&quot;Groß&quot;)
</code></pre>

<pre><code>## [1] &quot;GROSS&quot;
</code></pre>

<p><strong>In our upcoming blog posts we will present some exciting
features of <code>stringi</code>.</strong>
They are definitely worth to be discussed separately! Stay tuned.</p>

<h3>Performance</h3>

<p>And some benchmarks.</p>

<ul>
<li>String sorting:</li>
</ul>

<pre><code class="r">set.seed(123L)
library(microbenchmark)
x &lt;- stri_rand_strings(1e+05, 10)  # 10000 random ASCII &#39;words&#39; of length 10 each
head(x, 5)
</code></pre>

<pre><code>## [1] &quot;HmPsw2WtYS&quot; &quot;xSgZ6tF2Kx&quot; &quot;tgdzehXaH9&quot; &quot;xtgn1TlDJE&quot; &quot;8PPM98ESGr&quot;
</code></pre>

<pre><code class="r">microbenchmark(sort(x), stri_sort(x))
</code></pre>

<pre><code>## Unit: milliseconds
##          expr    min     lq median   uq    max neval
##       sort(x) 1050.4 1062.8 1076.1 1110 1176.6   100
##  stri_sort(x)  234.2  239.7  243.5  250  303.7   100
</code></pre>

<ul>
<li>String joining:</li>
</ul>

<pre><code class="r">microbenchmark(paste(x, collapse = &quot;, &quot;), stri_paste(x, collapse = &quot;, &quot;))
</code></pre>

<pre><code>## Unit: milliseconds
##                            expr   min    lq median    uq    max neval
##       paste(x, collapse = &quot;, &quot;) 45.21 45.70  46.64 53.15 244.28   100
##  stri_paste(x, collapse = &quot;, &quot;) 10.14 10.44  10.70 16.36  18.71   100
</code></pre>

<ul>
<li>Searching for a fixed pattern:</li>
</ul>

<pre><code class="r">set.seed(123L)
y &lt;- stri_rand_strings(10000, 10, &quot;[ACGT]&quot;)  # 10000 random &#39;genomes&#39; of length 10
head(y, 5)
</code></pre>

<pre><code>## [1] &quot;CTCTTAGTGC&quot; &quot;TCGGATAACT&quot; &quot;TGGTGGGGCA&quot; &quot;TTGTACTACA&quot; &quot;ACCCAAACCT&quot;
</code></pre>

<pre><code class="r">microbenchmark(grepl(&quot;ACCA&quot;, y), grepl(&quot;ACCA&quot;, y, fixed = TRUE), grepl(&quot;ACCA&quot;, 
    y, perl = TRUE), stri_detect_fixed(y, &quot;ACCA&quot;), stri_detect_regex(y, &quot;ACCA&quot;))
</code></pre>

<pre><code>## Unit: microseconds
##                            expr    min     lq median     uq     max neval
##                grepl(&quot;ACCA&quot;, y) 4928.0 4968.9 4987.0 5008.9 12723.2   100
##  grepl(&quot;ACCA&quot;, y, fixed = TRUE)  899.0  906.9  912.0  919.2  2441.2   100
##   grepl(&quot;ACCA&quot;, y, perl = TRUE) 2145.7 2155.5 2162.8 2174.6  9707.1   100
##    stri_detect_fixed(y, &quot;ACCA&quot;)  514.9  523.0  532.2  558.6   893.4   100
##    stri_detect_regex(y, &quot;ACCA&quot;) 3720.2 3750.8 3805.6 3891.6  7411.8   100
</code></pre>

<ul>
<li>Determining a substring:</li>
</ul>

<pre><code class="r">microbenchmark(substr(y, 2, 4), stri_sub(y, 2, 4))
</code></pre>

<pre><code>## Unit: microseconds
##               expr   min    lq median     uq  max neval
##    substr(y, 2, 4) 908.8 915.4  920.3  945.4 3640   100
##  stri_sub(y, 2, 4) 924.4 945.4  955.4 1007.5 2476   100
</code></pre>

<p>As a <strong>rule of thumb</strong>: <code>stringi</code> functions should often be
faster than the R ones for long ASCII and UTF-8 strings.
They often have poorer performance for short 8-bit encoded ones.</p>

<h3>More information</h3>

<p>For <strong>more information</strong> check out the <code>stringi</code>
package <a href="http://www.rexamine.com/resources/stringi/">website</a>
and its <a href="http://docs.rexamine.com/R-man/stringi/stringi-package.html">on-line documentation</a>.</p>

<p>For <strong>bug reports and feature requests</strong> visit our <a href="https://github.com/Rexamine/stringi/">GitHub profile</a>.</p>

<p>In the future versions of <code>stringi</code> we plan to include:</p>

<ul>
<li>rule-based number formatting (number spell-out, e.g. <code>123 -&gt; one hundred twenty three</code>);</li>
<li>date and time formatting/parsing;</li>
<li>access to the Unicode Character database;</li>
<li>functions to read/write text files (with automatic encoding detection);</li>
<li>and many more.</li>
</ul>

<p><strong>Any comments and suggestions are warmly welcome.</strong></p>

<p>Have fun!</p>

<h2></h2>

<p><a href="http://gagolewski.rexamine.com">Marek Gagolewski</a></p>

<h3>Change-log</h3>

<p><strong>Notable changes</strong> since the previous CRAN release (v0.1-25):</p>

<ul>
<li><p>[IMPORTANT CHANGE] <code>stri_cmp*</code> now do not allow for passing <code>opts_collator=NA</code>.
From now on, <code>stri_cmp_eq</code>, <code>stri_cmp_neq</code>, and the new operators
<code>%===%</code>, <code>%!==%</code>, <code>%stri===%</code>, and <code>%stri!==%</code> are locale-independent operations,
which base on code point comparisons. New functions <code>stri_cmp_equiv</code>
and <code>stri_cmp_nequiv</code> (and from now on also <code>%==%</code>, <code>%!=%</code>, <code>%stri==%</code>,
and <code>%stri!=%</code>) test for canonical equivalence.</p></li>
<li><p>[IMPORTANT CHANGE] <code>stri_*_fixed</code> search functions now perform
a locale-independent exact (bytewise, of course after conversion to UTF-8)
pattern search. All the Collator-based, locale-dependent search routines
are now available via <code>stri_*_coll</code>. The reason for this is that
ICU USearch has currently very poor performance and in many search tasks
in fact it is sufficient to do exact pattern matching.</p></li>
<li><p>[IMPORTANT CHANGE] <code>stri_enc_nf*</code> and <code>stri_enc_isnf*</code>function families
have been renamed to <code>stri_trans_nf*</code> and <code>stri_trans_isnf*</code>, respectively.
This is because they deal with text transforming, and not with character
encoding. Moreover, all such operation may be performed by
ICU&#39;s Transliterator (see below).</p></li>
<li><p>[IMPORTANT CHANGE] <code>stri_*_charclass</code> search functions now
rely solely on ICU&#39;s UnicodeSet patterns. All previously accepted
charclass identifiers became invalid. However, new patterns
should now be more familiar to the users (they are regex-like).
Moreover, we observe a very nice performance gain.</p></li>
<li><p>[IMPORTANT CHANGE] stri_sort now does not include NAs
in output vectors by default, for compatibility with <code>sort()</code>.
Moreover, currently none of the input vector&#39;s attributes are preserved.</p></li>
<li><p>[NEW FUNCTION] <code>stri_trans_general</code>, <code>stri_trans_list</code> gives access
to ICU&#39;s Transliterator: may be used to perform very general
text transforms.</p></li>
<li><p>[NEW FUNCTION <code>stri_split_boundaries</code> utilizes ICU&#39;s BreakIterator
to split strings at specific text boundaries. Moreover,
<code>stri_locate_boundaries</code> indicates positions of these boundaries.</p></li>
<li><p>[NEW FUNCTION] <code>stri_extract_words</code> uses ICU&#39;s BreakIterator to
extract all words from a text. Additionally, <code>stri_locate_words</code>
locates start and end positions of words in a text.</p></li>
<li><p>[NEW FUNCTION] <code>stri_pad</code>, <code>stri_pad_left</code>, <code>stri_pad_right</code>, <code>stri_pad_both</code>
pads a string with a specific code point.</p></li>
<li><p>[NEW FUNCTION] <code>stri_wrap</code> breaks paragraphs of text into lines.
Two algorihms (greedy and minimal-raggedness) are available.</p></li>
<li><p>[NEW FUNCTION] <code>stri_unique</code> extracts unique elements from
a character vector.</p></li>
<li><p>[NEW FUNCTIONS] <code>stri_duplicated</code> any <code>stri_duplicated_any</code>
determine duplicate elements in a character vector.</p></li>
<li><p>[NEW FUNCTION] <code>stri_replace_na</code> replaces <code>NA</code>s in a character vector
with a given string, useful for emulating e.g. R&#39;s <code>paste()</code> behavior.</p></li>
<li><p>[NEW FUNCTION] <code>stri_rand_shuffle</code> generates a random permutation
of code points in a string.</p></li>
<li><p>[NEW FUNCTION] <code>stri_rand_strings</code> generates random strings.</p></li>
<li><p>[NEW FUNCTIONS] New functions and binary operators for string comparison:
<code>stri_cmp_eq</code>, <code>stri_cmp_neq</code>, <code>stri_cmp_lt</code>, <code>stri_cmp_le</code>, <code>stri_cmp_gt</code>,
<code>stri_cmp_ge</code>, <code>%==%</code>, <code>%!=%</code>, <code>%&lt;%</code>, <code>%&lt;=%</code>, <code>%&gt;%</code>, <code>%&gt;=%</code>.</p></li>
<li><p>[NEW FUNCTION] <code>stri_enc_mark</code> reads declared encodings of character strings
as seen by <code>stringi</code>.</p></li>
<li><p>[NEW FUNCTION] <code>stri_enc_tonative(str)</code> is an alias to
<code>stri_encode(str, NULL, NULL)</code>.</p></li>
<li><p>[NEW FEATURE] <code>stri_order</code> and <code>stri_sort</code> now have an additional argument
<code>na_last</code> (defaults to <code>TRUE</code> and <code>NA</code>, respectively).</p></li>
<li><p>[NEW FEATURE] <code>stri_replace_all_charclass</code> now has <code>merge</code> arg
(defaults to <code>FALSE</code> for backward-compatibility). It may be used
to e.g. replace sequences of white spaces with a single space.</p></li>
<li><p>[NEW FEATURE] <code>stri_enc_toutf8</code> now has a new <code>validate</code> arg (defaults
to <code>FALSE</code> for backward-compatibility). It may be used in a (rare) case
in which a user wants to fix an invalid UTF-8 byte sequence.
stri_length (among others) now detect invalid UTF-8 byte sequences.</p></li>
<li><p>[NEW FEATURE] All binary operators <code>%???%</code> now also have aliases <code>%stri???%</code>.</p></li>
<li><p><code>stri_*_fixed</code> now use a tweaked Knuth-Morris-Pratt search algorithm,
which improves the search performance drastically.</p></li>
<li><p>Significant performance improvements in <code>stri_join</code>, <code>stri_flatten</code>,
<code>stri_cmp</code>, <code>stri_trans_to*</code>, and others.</p></li>
</ul>

<p>Refer to <a href="https://raw.githubusercontent.com/Rexamine/stringi/master/NEWS">NEWS</a>
for a complete list of changes, new features and bug fixes.</p>

</body>

</html>

